{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "I ran the following notebook in a docker container with the following commands:\n",
    "\n",
    "```\n",
    "docker run -it -v `pwd`:/space/ -p 8888:8888 -p 6006:6006 --name dl -w /space/ --rm utensil/dl:models_notop jupyter notebook --ip=0.0.0.0` to start.\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "pip2 install -U pip\n",
    "pip2 install -U Pillow\n",
    "pip2 install -U captcha\n",
    "pip2 install -U tensorflow\n",
    "pip2 install -U theano\n",
    "pip2 install -U keras\n",
    "pip2 install -U tflearn\n",
    "pip2 install -U pandas\n",
    "pip2 install -U scikit-learn\n",
    "pip2 install -U h5py\n",
    "pip2 install -U pydot-ng"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "\"\"\" Convolutional Neural Network for MNIST dataset classification task.\n",
    "References:\n",
    "    Y. LeCun, L. Bottou, Y. Bengio, and P. Haffner. \"Gradient-based\n",
    "    learning applied to document recognition.\" Proceedings of the IEEE,\n",
    "    86(11):2278-2324, November 1998.\n",
    "Links:\n",
    "    [MNIST Dataset] http://yann.lecun.com/exdb/mnist/\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import division, print_function, absolute_import\n",
    "\n",
    "import tflearn\n",
    "from tflearn.layers.core import input_data, dropout, fully_connected\n",
    "from tflearn.layers.conv import conv_2d, max_pool_2d\n",
    "from tflearn.layers.normalization import local_response_normalization\n",
    "from tflearn.layers.estimator import regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learn = tf.contrib.learn\n",
    "tf.logging.set_verbosity(tf.logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST-data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = learn.datasets.load_dataset('mnist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = mnist.train.images\n",
    "labels = np.asarray(mnist.train.labels, dtype=np.int32)\n",
    "test_data = mnist.test.images\n",
    "test_labels = np.asarray(mnist.test.labels, dtype=np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_examples = 10000\n",
    "data = data[:max_examples]\n",
    "labels = labels[:max_examples]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def display(i):\n",
    "    img = test_data[i]\n",
    "    plt.title('Example %d. Label: %d' % (i, test_labels[i]))\n",
    "    plt.imshow(img.reshape((28,28)), cmap=plt.cm.gray_r)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAEKCAYAAAAy4ujqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztfXuQdGlZ3++Zme7p2/TM930L+xUsrDHEwiSSjRG0xMRR\nEqXUZC2TVUTDzVikRGOJFUH9Y79VqwSqRMVoKiBQrJdC8bZgaUQjU5GkUES5hUVEXGCF/dzdb+7d\nM9OXN390/84+55n3nO65dZ+efn5Vb53TPX15+8z5vc/9eSWEAIfDMV9YmPYEHA7H5OHEdzjmEE58\nh2MO4cR3OOYQTnyHYw7hxHc45hBO/BmFiLxYRP5k2vOYBM7yW+fpOp0ETvwIROQhEWmJyI6I7A6P\nb5j2vCI4VRKGiNwjIv9HRPZF5I8jf79LRP58+Pf3i8g/M39/rYg8JiKPishrT/C994rIL51mzjjl\nbz3pe0VkQUR+QkT+bvh//4CINM/w3YXE0rQnUFAEAN8YQnjPtCdyQXgcwE8DeCaAr9V/EJESgN8B\n8HoA/x3AfwbwgIg8I4TQFZGXA/h3AL5k+JY/EpG/CSG8cczvLnrG2I8B+AoAXx5CeFhE/jGAgynP\n6dzhEj8bEn1S5BdE5B3q8WtF5A+H52si8i4R+XsReXx4/lT12veIyI8Ppe2uiDwgIldF5JdFZFtE\n/lREnq5e3xeR7xORvxl+5usyJyvyTBF59/B7HxSRe7JeG0L44xDCbwD4fOTP6wAWQwhvCCF0Qgg/\nN7wWXCBeBOCnQgifDyF8HsBPAXhJ1neNCxF5lYh8cihlPyoi32xesiAibxCRLRH5mIh8rXpvU0R+\nUUQ+JyKfHV7j6P9vxBzWAHw/gO8OITwMACGEj4UQjs704woIJ/7J8YMAvkREXiQi/xLASzEgAzC4\nnm8B8DQATwfQAvDfzPu/DcB3AHgKgGcA+L8A3gzgCoCPA7jXvP6bAXzpcNwtIi+zExKRGoB3A/hl\nALcB+HYAPy8iX3yK3/dPAHzYPPfh4fP8+4fU3z6k/nYWfBLAc0MITQD3AfhlEbld/f3Lh6+5BuAG\ngN8aEhUA7gdwBOALAfxzAP8GwH+KfclwMf6hjDl8CYAOgHtE5PMi8nER+Z6z/axiwomfjd8RkVsi\nsjk8fhcAhBDaAL4TA1X5fgDfO5R8CCHcCiH8dgjhMISwD+AnAfwr87lvDSE8FELYBfD7AP4mhPCe\nEEIfwDswuHE1XhNC2B5KoJ/BgNQW3wTgb0MI94cBPgjgtwD8h1P87gaAbfPcNoCVjL9vD587E0II\nvxlCuDk8fweAvwbwHPWSm0MtpBdC+HUAfwXgG0XkyQCeD+AHQggHIYTHkH2dEEL4tyGELM3pDgBr\nAP4RgDsB3APghog876y/r2hwGz8bd2fZ+CGEPxeRTwF4EgZkBQCISBWDm+7rMbiBBEBDRCQ8UQ11\nU31UO/LYkuhhdf5pDDQFizsBfIWI3OJUACwCOI0jbQ+AdWY1Aexm/L05fO5MEJEXAfgBAF8wfKqO\ngfZC/J15C6/FnQBKAD4/1O5lOD5zimm0MfBB3DdU7z8iIm8H8A0A/tcpPq+wcImfjUwbUUReAaAM\n4HMAXqX+9IMYSItnhxDW8IS0P7G9qfA0df704XdafBbARgjh6nBcCSE0QwivOMX3/T8AzzLPPQvA\nR9XftZf/ruFzp8bQr/FGAN8znPuV4Wfq6/ZU8zZei89i4Hy7pn77WgjB/oZxYE2cSwsn/gkhIl8E\n4McxsNNfBOCHRIQ32QoGUmNHRK5iYIueFf916DR8GgaOp7dHXvO7AL5IRL5TRJZEpCQiXyYiz8z4\nDQsisoyBpFwUkWURofa3AaA3dCqWReR7MZCC1H7uB/BKEXmKiDwFwCsBvPUEv4ffx1HGQLr3ATw2\nnNtLAfxT877bh3NaGjounwng90IIj2Dg3/hpEVmRAb5QRKyJNRIhhE8B+BMAPzr87V+MgU/mXSf9\nrKLDiZ+Ndw09zBy/KSJUn38yhPDREMInAfwIgF8ahsF+BkANwGMYOO1+z3zmaUJZDwD4AIC/wOAG\nfIt9QQhhD8DXAXgBBlLwcwBeg4FWEsN/xGCB+nkAX4WBE/KNw8/qYOBQfDGATQw89neHELrDv/+P\n4Tw+goGEfFcI4U384GG04rk5v+cFw+9rDefwyRDCgxiED98H4BEMnIXvNe97Hwba1GMYLLz/PoSw\nOfzbi4a/9WMAbmFgfl2PfbmI/J6IvDpnft+Ogbnx+PB3/mgIYSPn9TMJ8UYcxYWI9AE8YyiJHI5z\ng0t8h2MO4cQvNlwdc1wIXNV3OOYQZ5L4IvL8YXbTJ0TkVaPf4XA4ioBTS3wRWQDwCQDPw8CL/H4A\nLwghfNy8zlUKh2NKCCFEc0jOIvGfA+CvQwifHoaA3g7g7owvT8a9996bely04fO7vPMr8twuYn55\nOAvxn4pB1hTxMI5nVzkcjgLiLMSPqRCu1jscM4CzFOk8jEG+NHEH4nnkuHHjRnK+trYWe0lhsL6+\nPu0p5MLnd3oUeW7A2ee3sbGBjY2NsV57FufeIgalkc/DoKHDnwH49jBIv9SvC6f9DofDcXqICEKG\nc+/UEj+E0BsWcLwbA5PhzZb0DoejmLjwBB6X+A7HdJAn8T1l1+GYQzjxHY45hBPf4ZhDeM+9S4Zx\nsrnyzoGBbbiwsAARSYZ+TPA89pyj2HDiXzKEENDr9aKj3+8nR3vOISJYWlpKxuLi4rHHekHIWhQc\nxYYT/5KBxO90OqnR7XaT816vh263i263e+xcRLC8vIzl5WWUy+Vjx1KphIWFhdQAkJDfMRtw4l8y\naOIfHh4m4+joKDl2Op3kaMfCwgKq1SpqtRqq1Woyut0uQgjo9/tYXFxMDcKl/uzAiX/JYInfbrfR\nbrdxcHCQHO1CoBeHxcVFNBoNNBoN1Ot1NBqNhPQLCwsIIWBpaSnlG3DCzx6c+JcMMeLv7++j1Wol\nx4ODg8yxuLiI1dVVrK6u4vDwEL1eLyH90tISk0KOOQK5KDhmA078S4Ys4u/t7WF3dzchf7vdRqvV\nOnZeKpXQbrdxeHiYSHo6/MrlclS1X1xcdNLPGJz4lwwx4rdaLezu7mJ7ezshf2zs7e2hXC7j6Ojo\nGOmXl5dRq9VQKpWS76K07/f7TvwZgxN/xmBj7vZIx93h4WFi11Pa7+zsYHd3F3t7e5nkpwd/eXk5\ncfJp30C5XE4Iv7i4mAoFOvlnB078GQRJFjuS7FTdtTQn6WnnHx4eJqE+TVza8Iz1MxR4dHSEo6Oj\nxKZnjN9JP3tw4s8YSEqbkMMjiU/bXhOfg448evL5Xv0dMdIfHh5ieXk5IT1j/0782YMTfwZB1Von\n4PBI0muJT8LTvifhGcunxCc08WMSf3FxEaVSKZUVOE6DR0dx4MSfMWiJTyceidntdlMS36r6tO3t\ne7Sqr9V8Li5M7iHxl5aW0Ol0EvK7xJ89OPFnEJqU3W43lY1nbXwt9Xd3d9FqtaI5/CeR+KVSKaXm\nu3Nv9uDEnzHEJD5Dd0dHR4m01xJf2/etVuuYdM+q3tN5/CQ9iV8ul93Gn2E48WcQMYlPaT/Kq99u\ntwEcL6nVKbda1deLC7+HpHfizy6c+AXDKAJpMupcfI6dnZ1Udh41AW3H0yuvK+z4uFKpoF6vo1qt\nolKpoFwuJ1V5+j1ejjvbcOIXGHYRoPpN6d5qtRKHHY87OzvY3t5OvPcstS2Xy6hWq0ldfalUStXZ\n83GlUjlWpKNHvV5PKveWl5eT9+lUXkfx4cQvKLIy8yjtNfFJ9p2dnWQBaLVaKeKXSqWErKyt14PP\nVSoV1Gq13MFSXb5naWnJ6/FnDE78AiKvRZaW+Pv7+9jd3cXW1ha2trawubmZitOT+AsLCwnxRSRR\n4/UY9zkOLiCU+E782YITv6DI8rZbic/im1u3buHxxx9PhevoD6DE5wIQk+JU4WnX28479lwP3ZLL\nMRtw4hccdgHII/5jjz2GVqt1zPlGwlNKW7t9ZWUlOa9UKokU18fYc9pHwBZcjtmAE7+AyIqr56n6\nt27dwqOPPoqDg4PEC6+98Xxcq9XQbDZTY3V1NTmvVCpJSy16++2IPe+q/mzBiV9gWGmvQ3naube9\nvY3NzU089thjODw8TBxwDN1pr36j0UCz2cTa2loyrly5kpxXKpVjXXTzuup6WG824cQvGGz1na3A\nY519LDW31WolRTTlchkhhETaMz6/srKClZWVlMTXjyuVSnReltTeS3+24cQvGGyqrB2xenpdVsve\neLTpSXhK+maziUajkTjy6JmPOeeyCO1En3048QsGqvS2+IbnjNMzK88Snz3wSHx20SHx6cijOWCz\n8kaRetzFwVFsOPELCF0RR8lOFZ8Vdu12O1VPT0cgJT7Db5b4zWYzFbrLk/hAPrGd9LOLMxFfRB4C\nsA2gD6ATQnjOeUxqnmElvs3Hp8Q/japP+57SnjF7Tfxxyeykn22cVeL3AayHEDbPYzKOtI2v6+uZ\nhksb36r6WuLnqfrNZjNZEHT2HVX9PDjZLw/OSnyBb7V9rsiS+EzWiRGflXcMrWmJH7Px8zLvnNzz\ngbMSPwD4AxEJAN4YQnjTOcxprqGz8zTxbfusrGaZlPi08a2q32w2Uxl3dhdcx3zgrMT/yhDCIyLy\nJAB/KCIPhhDea19048aN5Hx9fR3r6+tn/NrLCyvxdaLO7u5u4tw7ODhAp9NJCK+z86jCk/gsuOHI\ny7xz8s8uNjY2sLGxMdZr5bw6p4jIvQB2QwivN88H784yPlqtFjY3N3Hr1q3okV109EaY+nGlUsH1\n69dx/fp13H777cfOG43GsdRb25jDcTkw3OcwupKf+r8sIjURaQzP6wC+DsBHT/t5jgG0c48hPF13\nT4lPFR9AaosrHarT4TrdMMN20nEpP384i6p/O4DfHtr3SwB+JYTw7vOZ1vzC2vjavmdbLZ3KCwzU\nfBJcN8sg8XXIzrbcsnn4jvnAqYkfQvhbAHed41wcOL7ppZX4rVbrGFm5fbWIpFpj6Z552omnCe9V\ndfMJz9wrGKzEt8Rvt9vH6uK5h12pVDpG/JjEz6q6c8wPnPgFg7XxrarfbrcT+52ddajqM3RniW9t\n/FhrbSf+fMGJXzAwnGdVfW5zfXBwgH6/n3TVAdLOPV2AoxtralWfcLLPL5z4E8ao0KbeJcduXaX3\nqGearq7Gs40wY7a9h+scgBN/qoj1zc/b3oqv1445bd/bTDxr0zschBN/Ssjqm6/bbGUtBAAS4lvy\n6wXAxusdDsKJPwXk9c3XO8/GCA8g5YmnVNfEj8XsXeo7NJz4U0LeLrV5CwChJb7eBktLfCe9IwtO\n/Ckj1kl3XDtfF9tY4nvra0ce3MU7BeRJ+lEDOO7cI/FjpbY2LdfhAJz4U0WWtB/Xzs9z7sVsfIeD\ncFV/wsgiOZtpsKMOY/l8PkZ6S3gdv9cLgBPfYeHEnwJIdJ2ow3PdWUcvApr8lvTc3lon8Dj5HXlw\n4k8YeqccbpJBgnc6naSXHolvSa/DeLq3Hvvr5RHf4SCc+FMA1Xu7aQb76OvW2VbiW6eebrel97TX\nxPeQnsPCiT9h6M0vdS6+3jQjS9UnslR9l/iOceHEnzCsqs8qPJJeq/pHR0cJ8fm+mCffSny9Rba3\nznbE4MSfAmISn6Snqq8lPk0DwvbP19tlVSqVVNGODus5HIQTf8LIU/WtxB9l4+s98rRX32buuVff\nYeHEnzA08a205x73lPxW1QeOJ+5Ylb9cLqcSd1zNd8Tg+t8UYDfMYHstvUWWdvJR3QfS6brjDCe9\nIwYn/oShm2nSsUdpr7fIsptiWhs/tgBkleJ6rr7Dwok/YWRtiqn3xsvaFBPIJ72V9C7xHVlwG3/C\nyNsGm3vjaYmvVX069zTpYza/t892jIITf8KwxI+p+trGp6pvU3bzbH0nvGMUnPgThlb1maJrVX0d\n1htX1dcSn6+LHR0OwIk/cWRJfO3VZyZfzLlnt76K2fh8ncORBSf+OWOcLcFtyi7Tc0l2a9cDSPXW\nsxl51rFXZOR1F7bHrL+dFnoxzNOItIl0Uo1pVhZcJ/6EkdVSSzflANJpucAThTmxIpxZy8qzrcNj\nx6wxLvnzSB6LetjnYhGSoi+qJ4ETf0qIkd468bS9zkVg1htt6CIl3XlIH21zEp53u92xiB/bE9D6\nRmLakn7OalT282YdTvwpQKuuMSkHIGWrhxCwtLSEEMKlkfh0cFpi6+YkHPqxTmTKQlY4M9ag1J7r\nIwc/c9aucx5GEl9E3gzgmwDcDCE8a/jcFQC/BuBOAA8B+NYQwvYFzvPSIUvNj0l8Db0DrrX1Z+Gm\n1BKfGYy2C5EuS7aj1+vlfv4o0nOzUU1s/Zj9C+3/4jKp+cB4Ev+tAH4OwP3quVcD+KMQwutE5FUA\nfnj4nGMERtn3/X4/0+YUkaS7ziw32tAS33Yh0puD8qjPu91u7mdbm91eQ3Yt0j0L9NAbkvLzaGad\n1blYJIwkfgjhvSJyp3n6bgBfPTx/G4ANOPFPBE16S35m38VszTxVf1YlvpbyzGFghSLP+bjT6eR+\nfsxBp89ZymwHr2lM0i8uLp7IsTgLOK2N/+QQwk0ACCE8IiJPOsc5XXrkSfyYpNHDEn8WG23EJH6s\nPFmP/f19tFqtE0n82FhaWkqak+otxTmXWN8Dqv6XCRNx7t24cSM5X19fx/r6+iS+durQEoLntpd+\nrM02bXt945HobLYxrdZa4+YpZA12EmanIXueRXwOLfFjvzcro1ETXzsODw8PE/IvLy+nciiyQqt8\nLhY9mCY2NjawsbEx1mtPS/ybInJ7COGmiFwH8Pd5L9bEnwdkJaFoFTfmvaatS7JT3WcLbdtQU++M\nWwSJb/sJ2oWt1+vh6OgoqT7MGlwI7MJwcHCAbrd7jGT6cV7F4uLiYhI2pKZhbfxKpYJ6vZ5ZINXr\n9Qob57dC9b777st87bjEl+Eg3gngJQBeC+DFAB444RwvPawqP6qfPknf6XRQKpWSxpqa+NVqFbVa\nLaXuTzuObzUA2zbcLmp56jyLk3Qmoz0n8YB49h0Xy1isnmYRfQu2PRl9KLa1uc6e7Pf7x2L9sxjn\nHyec96sA1gFcE5HPALgXwGsAvENEXgbgMwDuuchJzhpijjtdnJMVr9abaFDCWOJbiT9N4meZMroO\nwXrmSfr9/f3EbtfnlOqxMB+lb5aaHUvOiZ3nhfsqlcqxlmdW1dexfu2TmSWM49V/Ycaf/vU5z+VS\nIStUN0rVtzec3i3HEl87+CatZsZIb5uMHBwcJGo9jyQ5Ow3xyPPDw8NoJp/ObBwnVh+T5vSFxEww\nPq5UKlEnH/8PIpIsuNYRO0tef8/cO2fkeexjpLeFOpb4Wao+w1BFUfVjEj/WUozne3t7STWiHkdH\nR8euob6uQDxWH8vMs9l5VMmzFpV+v49KpXJMvdcOVh3a0983S6QHnPgXgizSj1L1tTp7UlW/CKSP\nSXySfWdnBzs7O0mXoazR6XRyJXpego7NfVhaWkqiJLxOIYRcU6JSqUQlPZ2rmuSzHOd34l8Qsopw\nslR9nZJqJb7eMCNP1Z8U+bPU/CxVf3d3F9vb29ja2koWAL0Q6HPtdIup7NaTHkvYIck16Xu9HpaW\nltDv9zMdh0dHR6hUKgCOS3pGVJaWlo5pF/zcWYIT/4IwKi03qzItK3NM75hjCX+epB8ltfJ+T7/f\nT3UTohTf2dnB9vZ2MmxTUb1zkA2X6X0DrFkTy9AbVYDDXH+dRET/iN7oxLZF29/fR6VSwcLCQqox\nChca+7/LQlGcgE78gmJUBpq94SeZwKNVY+uvaLVa2NzcxNbWVmpo0tNzDww85NVqNXGahRBSCUs6\nn15vFpIn8bMce9QCdF6A3p344OAAS0tLWFlZSdT6fr+Pw8ND7O3tYWFhAd1uF41GI5P8tvWZPS8K\nnPgFRKy4JJaQMknCEzZcZzsH7e3tJSTXhN/e3sbOzg7a7XbiwCTZeaxWq1hYWDiWP88jnZlZBTix\na2WPMeLrJCERScwpS3xdTUjSax8MFwNGDopMfid+QZEl8adNfkp8W1DDYVV7q+YfHR2lPo9qvH6s\n8+jt0RI/5ujLG71eL1UAZI9h2PuA5gGJ3+/3E5NES3qSXvdGtLkFRYz1O/ELiJgX2zq3pkV6Sny7\nEYi2660DT49ut5tZB8/KOUrc2DFG/Cwvf8wR2O12owuWrf7TvgxqNCEEHB4eAkhL+mq1mjQJidn4\nRSI84cQvKEZJfH0z8/UXCe2913H6GNm1U08fd3d3k1h5tVpNyK6JzVGr1aLHUqmUm32XF+Mn8TXZ\nbd2A3qk4djw4OEgkPRcp2w1Zz0H/P4sEJ37BYNXEItj4WorF4vQ7OzspJ5726DNJh+ecK9V72vaN\nRgMrKyuo1+uo1+uo1WrRY7lcziW8vm72MYDEAalJr4+6DLjX6yXSXu9krCV9o9FINAUSn4TXKr61\n+acNJ35BURQb36quVuJzP4CtrS3cunULW1tbx7Lx9KCKrD35lUoFKysruHLlCprNJur1OhqNRnLU\n5yS+vkYx8uvrqNHpdKJE56D3nqTXzr3t7e1UTkW9XsfKykp0/wOdwls00gNO/AtBLKkldrQpqYQl\nvG3+qOPZF0F+PT97botstDTf3t7G7u5uQiy7xbfOhKNqX6vV0Gg00Gw2sbq6itXV1UTqa8LzyCjA\nKCmfhaOjo2hTTYYMRSTRZhiaY9IPIxI294BRCvt/LSLhCSf+OSNG9qy20Vmdd0h8nbxC9VLHtXWS\nynndYLaennF6nu/v76fCdFTludEnvd60g6vVKhYXF7G8vJxI7CtXrhwbq6urWFlZQaPRQK1WQ61W\nO1Z6nCXZT/Lb9aLKa8uWWyR4VlakJXUsmSnrNUVL53XiXxB4Q9gUXS0d9I3C98SIH+sTZyX/eUp7\n3Q7LNsLc29tLEnTotGNJLZ1jdHBxcapUKsnvrVQqWF1dxdraWnJcW1tDs9lEs9lEo9FIefJjBIw5\nNMf9/TpKQuLrohwSXy+u+vrGCG+f04uERpE0ACf+BSArR992pYmRH0BKxadNTMmkJX6sxvw85s5E\nlViSy97eXiLxSXxK/FarhaOjo9SiZU0WEp9Ep4rfbDYT555uhcXYvW0vdpZohs2x1y22tMS3C6sl\neJ50L7K0B5z4F4Ks6rwsVT+r5NTaoHQsxWz981T1bZGNbpShE3Q08Snxe71eQhpbX1Aul1Gr1bCy\nspI5GLKzKbux33ka8ttra8tvs1R9fX3y1H0t9fn6IsKJf86I3RDjkp6I2fiaQHlOvvOYP3vjMVw3\nqpRW2/ghhEQ6k+gcdNrFvPU8r1QqxxyZHGdR8e31zSqvZfPNrF4HMWlvHaAxVb9oC4AT/4KQ14Aj\nbwEYJfFZqGJLVS/CxrdltZTyNkSnbXwRSarY6NzTaj0JrhcDfSyXy7m59mdNWOK1ZWquvdYkfszG\nz5L0ruo7AKRV/bzNIE9i42vHno3ln7eqbyU+Y9ibm5vY2dlJVP9Yzzw6y7TEZ4z+2rVriTqfNWLh\nuvMMV+qMOk16/h/Gkfh56n7RCU848S8AWap+VhgvdoPEEnhi+fonjeNn3Yx8nhJfd8TVXXR2d3dT\ncWzuYCsiSfRBtwljkgtj9M1mM1V4Y4fuXR/DeZKfUpwaQAghJeljexPq66z/v1qb099D1V/7CYoA\nJ/45Q4fmYt5923BjWtLBzpPntt237pZLwutaepavlsvlxCN/7dq1JDZP9Z659kXd+49kjmVKal8L\nFwogXbTEzTkODw/R6/VSFYdZ4b1pwol/Acjz6luJP42WTXl2KKVXjPRsf83XkrTlcjn53EqlkkrK\nsXZ9pVJJeesnvRMQYb9L+1fo/LP9+xgF0NoC/7e6L4H+n/Kzitaay4l/QYhJfC3tiyDxY7ZpFvG1\nxI+RgufVajVR6218nhJfZyNeRPuwUSBpeW7/FkuX1uE/LfH19dLlu3oBYb8/l/iXHFp11rHdmLS3\nqvak5pcVf9attLQUo8Q/OjpKHIx6A0o6xGjT04vPNFwSP5aHMC1VnwuAJn9MzdcSn+nIWuLr63Vw\ncJB8tk0UKhKc+BeEPInPFNFpSPyseen56c6/WtVvt9vodDpYWFhICLy8vJwqpWVM3g7a+MvLy9Eu\nutNQ9S3p+fxJiG+bc7JfAGsB+J4iSXvAiX8hyCN8nsSfxhyt8zEm8bWq3+12sby8DABJCm69Xk+p\n9Vm19LVaLUUM7V2fpKpPaO+8rfKLVUaWSqVUV96Yc0+nUttagCKR34l/zsjz6sds/GmQP4v0OtdA\nF+ZoVb/f76NerwN4oj9eo9HA6uoqrl69morTU8rrYTPwzpqQcx4YR9Unkfkaq+priW9JPy0nbh6c\n+BcATeKzJnjkJYyc1lSIxZ95TpveDi4CVG8BJB795eXlJFGHcXoObf/Tiz9NjLPA2KShrGQpm6eh\na/NtFMcl/hwgZjeeJAstS1Ow2zbzs7WWMerGtiEou4WXTr9lv3ndcMJ6xHVmYYzktqz1siCm2dn/\nWZHhxL9gWLXWFphkESJPDWdnGP0Z40qTrJAdj5b4tssMs9y0JNT7y1niX0Q9wTSRp83l5e0XDU78\nC0RW7XhsAdCwYUBLfv0ZOt10HMQcUrruXu9Tr9V89pTjZwDpunYt8W2HoMtCeiLP3Moy54pG/pHB\nUxF5s4jcFJEPq+fuFZGHReQvhuP5FzvN2cM4pI+RIc/xllfZNy60qq9jz7qXnlX19XZZ2tSgqh8r\nHY4R/zKRH8hX84tMemAM4gN4K4Cvjzz/+hDClw7H/zznec0sYjd3HumzFoE8r3tWoc+4N1hM4rMY\nh3X1WtVnMY7uIqsTVPJs/Gkm6FwkrI0/S2o+MIaqH0J4r4jcGfnT5Vq+LwCW6KPITuRJfLub7Elv\nMKvqZ3XaYf88Snyq+rqBhW0WQtLrNNzLqurzOK6dXzScZRl+hYh8UER+UURWz21GlxCjSD9K2sfI\nbyX+uKBzj6q+Tsm1xLcSn62y6dzTqn6exL+Mqn6eXR8L3xWN/Kd17v0CgB8LIQQR+QkArwfwXVkv\nvnHjRnIgn+htAAAXf0lEQVS+vr6O9fX1U37tbCAW+7W98W0HHQ6CpI7lzWsbe2FhIbU987iwN6o2\nIfTCEss2zFtwsha0opD+NASMSfgiYmNjAxsbG2O99lTEDyE8qh6+CcC78l6viT8PsGTXfdt148lY\n0wcSRGeEURXnTjSVSiXlZGNIbdwb0haQaImt5xYLxVkTxKb2Li8vp1R8nnOeRVkA8jDKW8/XAOkw\nbaw5in7NRcMK1fvuuy/zteMSX6BsehG5HkJ4ZPjwWwB89MSzvKTQNwEdX1pCanU45gQj8ohP0mtV\ne1yVP5aHbp1zdjGyXWZ1QpFN7T08PDxWrsvvLTKy7PYsG15jHPIXDSOJLyK/CmAdwDUR+QyAewF8\njYjcBaAP4CEAL7/AOc4cbDmmvlmsHWxj3nxdHvFJep08YzP68mAlvu3pl9ckgyTQxNfRAb1waKnI\nqrYiIssWz1sE9OuyCF9k8o/j1X9h5Om3XsBcLgWsxLcE0K2nbGrr0tJSkiSTR3wgvQ/d8vLy2Cmi\n2imniW8z77Jad1uJr3fd0R1orCp8Uh/EpBAjfVZsPua0s9LeEn5UBGda8My9C4AmvnXExTzf40p8\nHSrTdnkshz8PWV18u91udG++UYUpWuJXKpVjv7mIraeA0aQfFa4jrJqfRf4iwYl/ztA3u328uLiY\naeOTZCSIJb4mopb0lUolFWYbZ35Zqn6v18sNxwGjbXzbgUY7HovkDc8iPY95zr0s0lvy21EkOPEv\nALwB9DlJdnBwEPXq8+9Zqr7uWEPS63Tak8TzY159OiFHFdhoyadVfV3Caz9fZ/wVDTHS82hJn5cl\nmWXnFzGkCTjxzx1aAtjH/X7/mEMvth0WkG51fXh4mNxMbGkdS6cdR6patVSTn22hY/vy6ZvWhvS0\n9Ne1/boWne8rAvISbvQOwba+3jpqrTYXC89aH0lR4MS/QIik+7rFknpig+/r9/sJ8fkZpVIJtVot\nKaCJEWycedkRazZxGjW1aCq9hfVR2GH3ArSFSvx9NmRLE47NR4penejEvwDkqXeWZLa1U1YjRwCJ\nxsA8el0rr6XWuDdZjPDntQBYFGUx4HXlddOSvdPpHNv9V/ck0FqVDaeWy+WE9FkbhhSJ/E78C4Zd\nBCzpY5IfSN+gABIpVSqVktp5fUNqr75WRbPmdBKJn4dYbDsv5j1tUOJr34Q+7u7uHitUsh2ICFud\nqNuNxXIhigQn/gXBEp4EyFL3tTNNS3wgbe8vLS2NLfHzpP8o0o8j7WOELhLJY7CalG5CcnBwEFX1\n9XXOSqDSewFS4hfZznfiXwCsba+fz7PvraqvveckH4lvb0jrRDsJ+a0GEiP/KBRNsmdBa1K2gzC3\nBLeqPm18lkTzd1qJT/LH8jOKRHrAiX9hGIf8WYtAVuw4hBCV+HaTjqzv1nOw8edxHHz6ptewMfEi\nLwJa1WfeAXsR7O3tjaXqW4nPZCpKfFuJ6RJ/zhBz8uURnjeJDZXpc+YC6CYZWRI/tgjoYxbhx7Hz\nx1Hzi0j+mKrfbrcT0p9U1ddefdr3tuS6iB2InPjnjFGruk1usWoibzJt59tWWZRUtlFmu91GqVRK\nkVlL9VhugV6E+v1+SlLpnAParf1+P7mRtdqspaf+rXahod/iPK5x1jFvUdLqPZuPkPTb29uJxOei\n2u/3ExNL2/F6z0DdYLRcLucuokWBE3/CEEn3oueOM41GAysrKwghJOolicqbj04+3rxUUXd3d7G1\ntYVarZZoBTqJhOeaIJrwmih6EapUKsn86vU6Go0Ger1ecnMz4aXdbmNnZweLi4vodDpot9uo1WrJ\nsVarJQuWnkfs2oyDmA9Cn2dFGACg3W4nkn1nZwc7OzvY2trC9vY2tre3sb+/j4ODgySaUiqVUK1W\nISLJ/4tbgHNPQDr0tI+maES3cOJPGLxJtXpI4rfbbQBIpBHwROouz7kRhm6Qubu7m2xXxbRbPaia\n6hJbPrb2ON+zvLyckF4Tv9PpJM02mGDUarWSz+JCQNJrs4RbTPE62Osy7vWzKrRVp2PhRNr2Wq3f\n2dlJCE/yt1qtVBtzalDcCbhWq+Hq1atYXV1N9gm0vQZjvpGiwYk/YfDG1Q4hEp9ttbSkp4TXEp+q\nNYm/s7OTqJ/9fj8lrZkjz8VGS8gQQqqYSESOhaaq1WqyGy7nyJu83+8nRA8hoNPp4ODgAPV6PTE/\n7FZcnAO/M3Ycdf106NOeA+mUYptrT889pb0m/+bmZio9ms67crmcPK7VaonEX1lZQb1eTzz5ecQv\nGvmd+BOGVfVJfKr3seq82EKga/R1mmi/P9jUkumlAJLvsznm+jGf05149OaXJL72VFPCk/Ttdjup\nI7DEZw78WYnPBUz7IbRpIyKZPQL7/X6iIcUk/tbWFjqdTkrrsU1KGo1GpqpvNaoiEp5w4k8YWuJr\nVV/ngmuCt9vt5IbSfe60xOcNSnWbXn5rVlDl16q+Jr02Eyjxqeo3Gg20Wq1UWIqqPp2RVIlj0p7E\nL5fLyXWwjj+e50EXw2iHGs/1dbLNQ3u9XsqZZ4m/ubmZLJwLC4PeCUtLSymthxuD6i3BdezeOvKK\nugA48SeMmHNPh+Ridfg6jdeGofb395PX0Bwg6fk9bM5JbUJLfH0eQkgVnMQkvogcq8DT50wptqmw\nTJih2nxaclD11oN+BybYZBXgkPjWuaclPq8JfSMsimo2m1hbW0Oz2UyuRaPRSFR9a+PzdxUVTvwJ\nw0phTXoAifOO2WSlUim5mbgwUOLz75r0Vr0ngbXqr73fBM+txKe0o8OOUYeDg4NE1dchxsXFxWPS\nXu/GO4r4o8hCTUkPvQAtLCwcW5j0kRI/RvrNzc1E0tfrdQBPePWbzSauXbuGZrOZOFK5KGYRv8hw\n4k8YWtUnMXWBTa/XS2WT2c63ulEHnWlUMWOfr8NzDEvlkS7mf9BpqyKCdrudWoy4WHFeeT35rLmR\npfLHwMzFPK89iR8blPitViuVhqtTcG21HRc+reLbWL7dPWgW4MSfMCjxNcH0zdfpdFCv15PccQ6q\n25Sa2u7nAqC1CZsuSqLSax3LLKOnngsATRHtLGTKcKVSQbvdRrVaRavVSo4ikkhELRl1tVqepLea\nSOz6AU84OnUIj4tijPTUrBij52+pVCpYWVkBgCQWf9ttt+G2227D1atXsba2lnjvq9VqypGqPfmz\nQnjCiT9hWOLb3WcZF4+Rv9VqJXYy7X4tbXnj27ZZWv08Ojo65hHX7bApkWlLazOBXn8SneYIz6vV\nKgAk5LCSkd/D3zqOpLeLgPZj0NHJx0y60bv76lp7ah98HR13DGPWajWUy2VcvXoVV65cSbz3tOtJ\nfC7YdiOUWVDxCSf+FECV0rZxWlxcRLfbTUl5Sno+Z+v2SXze+JRklvSa+LpZBDUO2t42RZULk/ZL\nULpric/zEEIi3WO9+nXeQOyoobPvCC3xOTeaFYxMxBps6MVAay8kcq1WQ7fbxfLyMtbW1rC2tobV\n1dUkXt9oNFCtVlMLmPbku8R35MJKfD6nG1PGJD0XAUsQ7fADkOyjl5Un3ul0Ekmst+LiHDgfSnyt\nAVCKxwhP559NGY71FdTXInYOZDfB1L/b+gv4vHYm6qHDnPxNurZBRBLVX4fsrKpvE4dc4jtGgjeZ\nVnl1hhizy7IGgGjVnnaoxcpAqcZ3Op1EumnScw58H4nO5+k91yTX51wE6Fm3i09esco4pI+d2x73\nlP66WSaHLn7SjTK0NkLTRofqeCTxWX1n/SQu8R0jEfPA6+wyTfT9/f1E2rMIhzeytnN1rDx2I5Ig\nVH25WGjSMw7Om1ovAJybzsUn2fUiwLqCPGk+SjLmFdno9Fu7k6/t9R878ndRTa9UKkkOPo/aMcnr\nzvO86keX+I5M6Jsm6wbXZLdOPjaDoEpLMjKhJ1b9psNdummHXni4k46W+DE7nKm5VuJzcE42R14f\nRyEWqtPPA0gRXcfp9QYf+pyP6Y2vVquJc29lZSWx55mQo+vr9dGq9bOQrBODE3/CGCUZaIPrjDnd\nAgpAKlc+Vshj03p1/byWkJZYwMAjn1X5prUBHRnQkpcONv2czhrkMYvUsdi8PufvtfH5WOhONyix\neQq2MpKZefV6/ZhTMuacnHU48QsGrV7T3jw6OkpCdbabq93WOoSQeKq1ZsCsO6sS26YUzE+31W88\n9vv9VDquPrepx+c9rGqf9ZyOQmjfSQgh8drTcUf1Pqs7bsxfchngxC8YtKOvWq2mSM84uu7mandu\nYWhOE58VdFYqMs2WpKeaa6ve9DmAqNOMw+44Ywlpc+d1uu2ooT8HiDv/eJ20n0Lb4JVKJeWxZ4Ud\nHXc6Pl/UnvjnASd+wWAlvia9zlPP2tW20+mkvOgkPptLUjLbRpM0Lexna483F5O8XHhL+pi6HlPJ\nbZw9psJrh6QltFblte/CLlzUokj6mMS34bpZdN6Nwkjii8gdAO4HcB1AD8CbQghvEJErAH4NwJ0A\nHgLwrSGE7Quc61zAZs0BadJnSXqdkhuTtnxucXExJel1TN/2hScR9FFnzmWNLHtdJ9tYj3ssBBfL\nvONviKUd037nc7rSUNcrMDzHoSU+328bZl4m0gPjSfwugFeGED4oIg0AHxCRdwN4KYA/CiG8TkRe\nBeCHAbz6Auc6F9Akp63Kx7r2O7Y9k4ig1WrlquIikpBeE9qOLM+2dm7FkmvyHHbAE9543dfe1u3b\nuevnQgiZZohuk0Xi654CethQnc7B1xrTZbXzRxI/hPAIgEeG53si8iCAOwDcDeCrhy97G4ANOPHP\nDEp8eqIptSj1GIayzidiYWEh6ddH1VuH4GxKrW1kQROD0tEeGeaLqdm6YIawi4OV+Fr70HX8upZf\nDwCZXndqA9q5xzg9M/A00fWgxI81zLxsaj5wQhtfRL4AwF0A3gfg9hDCTWCwOIjIk859dnMIbady\nAbDJPTGnk3Vw6Xg/k2729vbQ7XYz21Zp30IskYVVdjEVW6eucj4x6Fx6TXq7c03WAJCKalQqlVQ/\nA3b4scS34Tqt3ejH+jfkJSHNOsYm/lDN/w0A3z+U/GPvlnDjxo3kfH19Hevr6yeY4nyBxNc98fSR\nzjXtDbepq7TDtU3NqjxKQ+AJEmqnGwnE79MLDyMGluxZ9nCMQNZZZz32eTkGWrOI9RzQFYE6AUo3\nC2UFni0k0qbTrGJjYwMbGxtjvVbGyaQSkSUAvwvg90MIPzt87kEA6yGEmyJyHcB7QghfHHlvGOc7\nHAPEyK6PBwcH2NnZSbWOsud2Vxie7+7uJnY+cLzllY4oxOx/EiSWg2/z8bW6rJ/r9Xq5El3vWhOr\nsFtYWMg1RSjdm81mUlKrH1er1WM9+7iAXKYEHSARAFFVZVyJ/xYAHyPph3gngJcAeC2AFwN44CyT\ndMRB6c2jjvPr6jrgiXZbtuuO3rSDXXCyhk58IQmoDQBI8vntgqFJbpt76GMIIdX5RmfT8TewHNhq\nBKw8jJFeZzuyyEYX2+hQZcxHchnt+DyME857LoDvAPAREflLAAHAj2BA+F8XkZcB+AyAey5yovMG\nEl2f64IaEj+P9JYM3BoqVt2nN5GwhNB1AaNseGoNsaw/JgDF6gWAJxKU8hYmxuL179Pn2ltvvfg6\ny9GaJ7NWXXdWjKXqn+kLXNU/Eey1sh5ynYij98/jc3rL59jQzS9tnNySO2+O1g7XqbKx7j66aUWM\n0Pq3xjzqHHTYWc98zFOvF0AOmioxB+VlWwDOQ9V3TAlW8pNYVtKzYy+lHiWfrZ7j7q9ZYTOSf9y8\neVuMQ6kdCxdmbTqhn4tFCvRgZV0W6bUvIubEO0mfgMsMJ/4MQN+QuokH7XGd9643rdS76OrdbbKe\ns1tCW4+/TaWNHUl87Rxkg1D23c8jtna0xRxwVorbRUCn3MZMDRujn0fSA078mQNV0SyVmR12rAnA\n85gpwO21bUhOl9LSxMhLp6XXXVcO6s0uqBnQkcfv0f4JnS6cFV3IUu11VWJWVCEWm5830gNO/MJh\nnJswL+RkU1g1aWKNNrVE1e/hlta2WIXnJLl1BFILiWXW2e/Oeo1OEY49jtnwPKc2lHU955HkMTjx\nLxliSS62lDXmMyDpWAqclTYb61irpX5M1Y+R3qrysc68+j2xc9vl1kk+Ppz4lxCW+Jro+jW67RYl\npiV4rGouy8Yn8bMcezGb3aYP29fmlQjbQiV7DRzZcOJfQpD0MemuFwVdmEPSxyS6fZzVJIPJPXnh\nPNvkInae1wjELhx5qcKObDjxLxmsqs/nbJhMS1ebHms749hGG7HutjzXmkTWMRay08/Z92R9jv77\nZa2iuyg48S8hdIlsLD6um2TG1PasZpm6IMg2++C5/q6846gx6v2x5Bsn/fhw4l8yxEJYOvQ3qoml\nJXXsmDeyQmix4p28x1nFPnmf6xgfTvxLCFvSS5U/1h1Hn4+qDLTH2HlWSW7W8SSvzTu6tD8ZnPiX\nDO7gcowD148cjjmEE9/hmEM48R2OOYQT3+GYQzjxHY45hBPf4ZhDOPEdjjmEE9/hmEM48R2OOYQT\n3+GYQzjxHY45hBPf4ZhDOPEdjjmEE9/hmEM48R2OOYQT3+GYQzjxHY45hBPf4ZhDjCS+iNwhIn8s\nIh8TkY+IyPcNn79XRB4Wkb8Yjudf/HQdDsd5QEbtXS8i1wFcDyF8UEQaAD4A4G4A3wZgN4Tw+hHv\nD6O+w+FwnD+GW6xHmy+ObLYZQngEwCPD8z0ReRDAU/nZ5zZLh8MxMZzIxheRLwBwF4A/HT71ChH5\noIj8ooisnvPcHA7HBWFs4g/V/N8A8P0hhD0AvwDgH4YQ7sJAI8hV+R0OR3EwVl99EVnCgPS/FEJ4\nAABCCI+ql7wJwLuy3n/jxo3kfH19Hevr66eYqsPhyMPGxgY2NjbGeu1I5x4AiMj9AB4LIbxSPXd9\naP9DRH4AwLNDCC+MvNedew7HFJDn3BvHq/9cAP8bwEcAhOH4EQAvxMDe7wN4CMDLQwg3I+934jsc\nU8CZiH8OX+7EdzimgDzie+aewzGHcOI7HHMIJ77DMYdw4jsccwgnvsMxh3DiOxxzCCe+wzGHcOI7\nHHMIJ77DMYdw4jsccwgnvsMxh3DiOxxziIkTf9x64WnB53c2FHl+RZ4bMNn5OfENfH5nQ5HnV+S5\nAZec+A6HY/pw4jscc4iJNOK40C9wOByZmFoHHofDUTy4qu9wzCGc+A7HHGJixBeR54vIx0XkEyLy\nqkl977gQkYdE5EMi8pci8mcFmM+bReSmiHxYPXdFRN4tIn8lIn8wzd2LMuZXmI1UI5u9/pfh84W4\nhtPejHYiNr6ILAD4BIDnAfgcgPcDeEEI4eMX/uVjQkQ+BeBfhBA2pz0XABCRrwKwB+D+EMKzhs+9\nFsDjIYTXDRfPKyGEVxdofvdijI1UJ4GczV5figJcw7NuRntWTEriPwfAX4cQPh1C6AB4OwY/skgQ\nFMj0CSG8F4BdhO4G8Lbh+dsAfPNEJ6WQMT+gIBuphhAeCSF8cHi+B+BBAHegINcwY34T24x2Ujf6\nUwF8Vj1+GE/8yKIgAPgDEXm/iHz3tCeTgSdz05LhLkZPmvJ8YijcRqpqs9f3Abi9aNdwGpvRTor4\nsRWsaHHErwwhfBmAb8Dgwn/VtCc0gyjcRqqRzV4Ldd9NazPaSRH/YQBPV4/vwMDWLwy4D+BwM9Df\nxsA8KRpuisjtQGIj/v2U55NCCOFRtW3SmwA8e5rziW32igJdw6zNaCdxDSdF/PcDeIaI3CkiZQAv\nAPDOCX33SIhIbbjyQkTqAL4OwEenOysAA01Ja0vvBPCS4fmLATxg3zBhpOY3JBLxLZj+NXwLgI+F\nEH5WPVeka3hsfpO6hhPL3BuGJX4Wg8XmzSGE10zki8eAiPwDDKR8wGDr8F+Z9vxE5FcBrAO4BuAm\ngHsB/A6AdwB4GoDPALgnhLBVoPl9DcbYSHVC88va7PXPAPw6pnwNz7oZ7Zm/31N2HY75Q2HCVw6H\nY3Jw4jsccwgnvsMxh3DiOxxzCCe+wzGHcOI7HHMIJ77DMYdw4jscc4j/D7SSVNeN6b40AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd40e30a390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting mnist/train-images-idx3-ubyte.gz\n",
      "Extracting mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Data loading and preprocessing\n",
    "import tflearn.datasets.mnist as mnist\n",
    "X, Y, testX, testY = mnist.load_data(one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_examples = 10000\n",
    "X = X.reshape([-1, 28, 28, 1])[:max_examples]\n",
    "testX = testX.reshape([-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Step: 157  | total loss: \u001b[1m\u001b[32m0.22930\u001b[0m\u001b[0m\n",
      "| Adam | epoch: 001 | loss: 0.22930 - acc: 0.9326 | val_loss: 0.13490 - val_acc: 0.9602 -- iter: 10000/10000\n",
      "Training Step: 157  | total loss: \u001b[1m\u001b[32m0.22930\u001b[0m\u001b[0m\n",
      "| Adam | epoch: 001 | loss: 0.22930 - acc: 0.9326 | val_loss: 0.13490 - val_acc: 0.9602 -- iter: 10000/10000\n",
      "--\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Building convolutional network\n",
    "network = input_data(shape=[None, 28, 28, 1], name='input')\n",
    "network = conv_2d(network, 32, 3, activation='relu', regularizer=\"L2\")\n",
    "network = max_pool_2d(network, 2)\n",
    "network = local_response_normalization(network)\n",
    "network = conv_2d(network, 64, 3, activation='relu', regularizer=\"L2\")\n",
    "network = max_pool_2d(network, 2)\n",
    "network = local_response_normalization(network)\n",
    "network = fully_connected(network, 128, activation='relu')\n",
    "network = dropout(network, 0.8)\n",
    "network = fully_connected(network, 256, activation='relu')\n",
    "network = dropout(network, 0.8)\n",
    "network = fully_connected(network, 10, activation='softmax')\n",
    "network = regression(network, optimizer='adam', learning_rate=0.01,\n",
    "                     loss='categorical_crossentropy', name='target')\n",
    "\n",
    "# Training\n",
    "model = tflearn.DNN(network, tensorboard_verbose=3, tensorboard_dir='./logs/')\n",
    "model.fit({'input': X}, {'target': Y}, n_epoch=1,\n",
    "           validation_set=({'input': testX}, {'target': testY}),\n",
    "           snapshot_step=100, show_metric=True, run_id='convnet_mnist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tflearn.layers.core import flatten\n",
    "\n",
    "IMAGE_STD_HEIGHT = 28\n",
    "IMAGE_STD_WIDTH = 28\n",
    "RGB_COLOR_COUNT = 1\n",
    "OPTIMIZER = tflearn.optimizers.AdaDelta(learning_rate=1.0, rho=0.95)\n",
    "# , epsilon=1e-08, use_locking=False, name='AdaDelta')# 'adam' # 'adadelta'\n",
    "# (lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0)\n",
    "\n",
    "# This is the same as keras default glorot_normal\n",
    "INIT = tflearn.initializations.xavier(uniform=False) # , seed=None, dtype=tf.float32)\n",
    "CLASS_COUNT = 10\n",
    "\n",
    "def conv_2d_specialized(incoming, nb_filter, filter_size):\n",
    "    return conv_2d(incoming, nb_filter, filter_size,\n",
    "        padding='valid',\n",
    "        activation='relu',\n",
    "        weights_init=INIT) #, regularizer=\"L2\")\n",
    "\n",
    "def create_cnn_layers():\n",
    "    shape = [None, IMAGE_STD_HEIGHT, IMAGE_STD_WIDTH, RGB_COLOR_COUNT]\n",
    "\n",
    "    # input_layer = Input(name='input', shape=shape)\n",
    "    input_layer = input_data(name='input', shape=shape)\n",
    "    # h = Convolution2D(22, 5, 5, activation='relu', dim_ordering=dim_ordering)(input_layer)\n",
    "    h = conv_2d_specialized(input_layer, 22, [5, 5])\n",
    "    POOL_SIZE = [2, 2]\n",
    "    # h = MaxPooling2D(pool_size=POOL_SIZE)(h)\n",
    "    h = max_pool_2d(h, POOL_SIZE, padding='valid')\n",
    "    h = local_response_normalization(h)\n",
    "    # h = Convolution2D(44, 3, 3, activation='relu', dim_ordering=dim_ordering)(h)\n",
    "    h = conv_2d_specialized(h, 44, [3, 3])\n",
    "    # h = MaxPooling2D(pool_size=POOL_SIZE)(h)\n",
    "    h = max_pool_2d(h, POOL_SIZE, padding='valid')\n",
    "    h = local_response_normalization(h)\n",
    "    # h = Dropout(0.25)(h)\n",
    "    h = dropout(h, 1-0.25)\n",
    "    # last_cnn_layer = Flatten()(h)\n",
    "    last_cnn_layer = flatten(h)\n",
    "    return input_layer, last_cnn_layer\n",
    "\n",
    "def create_single_digit_model():\n",
    "    input_layer, last_cnn_layer = create_cnn_layers()\n",
    "\n",
    "    # h = Dense(256, activation='relu')(last_cnn_layer)\n",
    "    h = fully_connected(last_cnn_layer, 256, activation='relu', weights_init=INIT)\n",
    "    # h = Dropout(0.5)(h)\n",
    "    h = dropout(h, 1-0.5)\n",
    "    # output_layer = Dense(CLASS_COUNT, activation='softmax', name='out')(h)\n",
    "    output_layer = fully_connected(h, CLASS_COUNT, activation='softmax', weights_init=INIT)\n",
    "    network = regression(output_layer, optimizer=OPTIMIZER,\n",
    "                     learning_rate=0.1,\n",
    "                     loss='categorical_crossentropy', name='out')\n",
    "    # model = Model(input_layer, output_layer)\n",
    "    model = tflearn.DNN(network, tensorboard_verbose=3, tensorboard_dir='./logs/')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Step: 471  | total loss: \u001b[1m\u001b[32m0.18873\u001b[0m\u001b[0m\n",
      "| AdaDelta | epoch: 003 | loss: 0.18873 - acc: 0.9393 | val_loss: 0.10362 - val_acc: 0.9705 -- iter: 10000/10000\n",
      "Training Step: 471  | total loss: \u001b[1m\u001b[32m0.18873\u001b[0m\u001b[0m\n",
      "| AdaDelta | epoch: 003 | loss: 0.18873 - acc: 0.9393 | val_loss: 0.10362 - val_acc: 0.9705 -- iter: 10000/10000\n",
      "--\n"
     ]
    }
   ],
   "source": [
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "model = create_single_digit_model()\n",
    "model.fit({'input': X}, {'out': Y}, n_epoch=3,\n",
    "           validation_set=({'input': testX}, {'out': testY}),\n",
    "           snapshot_step=100, show_metric=True, run_id='conv4captcha_mnist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sc = tf.get_variable_scope()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'is_training:0',\n",
       " u'Conv2D/W:0',\n",
       " u'Conv2D/b:0',\n",
       " u'Conv2D_1/W:0',\n",
       " u'Conv2D_1/b:0',\n",
       " u'FullyConnected/W:0',\n",
       " u'FullyConnected/b:0',\n",
       " u'FullyConnected_1/W:0',\n",
       " u'FullyConnected_1/b:0',\n",
       " u'Training_step:0',\n",
       " u'Global_Step:0',\n",
       " u'val_loss:0',\n",
       " u'val_acc:0',\n",
       " u'Accuracy/Mean/moving_avg:0',\n",
       " u'Accuracy/Mean/moving_avg/biased:0',\n",
       " u'Accuracy/Mean/moving_avg/local_step:0',\n",
       " u'AdaDelta/Crossentropy/Mean/moving_avg:0',\n",
       " u'AdaDelta/Crossentropy/Mean/moving_avg/biased:0',\n",
       " u'AdaDelta/Crossentropy/Mean/moving_avg/local_step:0',\n",
       " u'AdaDelta/Conv2D/W/AdaDelta:0',\n",
       " u'AdaDelta/Conv2D/W/AdaDelta_1:0',\n",
       " u'AdaDelta/Conv2D/b/AdaDelta:0',\n",
       " u'AdaDelta/Conv2D/b/AdaDelta_1:0',\n",
       " u'AdaDelta/Conv2D_1/W/AdaDelta:0',\n",
       " u'AdaDelta/Conv2D_1/W/AdaDelta_1:0',\n",
       " u'AdaDelta/Conv2D_1/b/AdaDelta:0',\n",
       " u'AdaDelta/Conv2D_1/b/AdaDelta_1:0',\n",
       " u'AdaDelta/FullyConnected/W/AdaDelta:0',\n",
       " u'AdaDelta/FullyConnected/W/AdaDelta_1:0',\n",
       " u'AdaDelta/FullyConnected/b/AdaDelta:0',\n",
       " u'AdaDelta/FullyConnected/b/AdaDelta_1:0',\n",
       " u'AdaDelta/FullyConnected_1/W/AdaDelta:0',\n",
       " u'AdaDelta/FullyConnected_1/W/AdaDelta_1:0',\n",
       " u'AdaDelta/FullyConnected_1/b/AdaDelta:0',\n",
       " u'AdaDelta/FullyConnected_1/b/AdaDelta_1:0']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tflearn.variables.get_all_variables()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Conv2D/W:0',\n",
       " u'Conv2D/b:0',\n",
       " u'Conv2D_1/W:0',\n",
       " u'Conv2D_1/b:0',\n",
       " u'FullyConnected/W:0',\n",
       " u'FullyConnected/b:0',\n",
       " u'FullyConnected_1/W:0',\n",
       " u'FullyConnected_1/b:0']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tflearn.variables.get_all_trainable_variable()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
